{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "vscode": {
     "languageId": "plaintext"
    }
   },
   "outputs": [],
   "source": [
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** Introductory Examples for the NLTK Book ***\n",
      "Loading text1, ..., text9 and sent1, ..., sent9\n",
      "Type the name of the text or sentence to view it.\n",
      "Type: 'texts()' or 'sents()' to list the materials.\n",
      "text1: Moby Dick by Herman Melville 1851\n",
      "text2: Sense and Sensibility by Jane Austen 1811\n",
      "text3: The Book of Genesis\n",
      "text4: Inaugural Address Corpus\n",
      "text5: Chat Corpus\n",
      "text6: Monty Python and the Holy Grail\n",
      "text7: Wall Street Journal\n",
      "text8: Personals Corpus\n",
      "text9: The Man Who Was Thursday by G . K . Chesterton 1908\n"
     ]
    }
   ],
   "source": [
    "from nltk.book import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import PlaintextCorpusReader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus_root = './11_4_data'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "wordlists = PlaintextCorpusReader(corpus_root, '.*')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['七剑下天山.TXT', '冰魄寒光剑.TXT', '大唐游侠传.TXT', '白发魔女传.TXT', '风云雷电.TXT']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wordlists.fileids()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3489\n",
      "230.71596445973057\n"
     ]
    }
   ],
   "source": [
    "with open('./11_4_data/风云雷电.TXT','r') as f:\n",
    "    str = f.read()\n",
    "    len(set(str))\n",
    "    len(str)/len(set(str))\n",
    "print(len(set(str)))\n",
    "print(len(str)/len(set(str)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22\n",
      "173\n",
      "154\n",
      "43\n"
     ]
    }
   ],
   "source": [
    "print(str.count('石家庄'))\n",
    "print(str.count('石元'))\n",
    "print(str.count('既然'))\n",
    "print(str.count('裂'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\u3000\\u3000方震说道：“当时我只觉得一阵晕眩，醒来之后，那人早已走了。” \\n\\u3000\\u3000石元道：“那十三辆大车的金银财宝呢？” \\n\\u3000\\u3000方震道：“并没有劫走。可是他留下了一封信，要我们给他送到指定的一处地方去。” \\n\\u3000\\u3000黑龙掸师道：“这倒是奇闻了，劫镖还有不用自己动手的。你们却又怎肯乖乖地听他吩 \\n咐？” \\n\\u3000\\u3000方震叹了口气道：“我们是实迫如此，不得不然。” \\n\\u3000\\u3000黑龙禅师道：“却是何故？” \\n\\u3000\\u3000方震道：“那天所有护送镖车的人，全部都给那厮打了一掌，甚至康大人的家人子女， \\n也给这厮在身上留下了标记。” \\n\\u3000\\u3000黑龙禅师道：“那你们岂不是都受了重伤了？”心想：“但若受了重伤，如何能搬运财 \\n宝？” \\n\\u3000\\u3000方震道：“不，这一掌只是隔衣留下掌印，当时并不觉得有内伤的迹象。” \\n\\u3000\\u3000黑龙掸师道：“事后呢？” \\n\\u3000\\u3000方震道：“事后也不怎样。这根本就不是内伤。” \\n\\u3000\\u3000黑龙禅师诧道：“你怎么越说我越糊涂了，既然没有受伤，你们就该送康大人回原籍才 \\n是。怎的却又肯乖乖的听那贼人的吩咐，把十三辆大车的财物，送到他所指定的地方？” \\n\\u3000\\u3000一直没有说话的那个玄经道人忽他说道：“在你们醒来的半个时辰之后，是不是每个人 \\n都觉得浑身痕痒，痛苦难当？” \\n\\u3000\\u3000杨大熊叫道：“是呀，你怎么知道？” \\n\\u3000\\u3000玄经道人道：“我刚才对这位方兄身上的掌印，还有点捉摸不透，现在则是可以断定 \\n了，这是奇门七绝掌，有七种不同的功效，或令人患上痨症，或令人发冷发热，或令'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "str[5394:6008]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('，', 44370), (' ', 25182), ('\\u3000', 24661), ('\\n', 24650), ('的', 20516), ('。', 15783), ('是', 15207), ('不', 12737), ('“', 12443), ('”', 12371), ('道', 12362), ('了', 10747), ('一', 10744), ('：', 10219), ('他', 9478), ('我', 9074), ('你', 8477), ('这', 6821), ('说', 6622), ('人', 6388), ('来', 6037), ('有', 4782), ('个', 4687), ('？', 4597), ('大', 4459), ('在', 4297), ('那', 4201), ('也', 4054), ('得', 3778), ('们', 3557)]\n"
     ]
    }
   ],
   "source": [
    "fdist = FreqDist(str)\n",
    "print(fdist.most_common(30))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "词频在0~99的词数量： 2669\n",
      "词频在100~999的词汇量： 678\n",
      "词频在1000~4999的词汇量： 121\n",
      "词频在5000及其以上的词数量： 21\n"
     ]
    }
   ],
   "source": [
    "from collections import Counter\n",
    "W = Counter(str)\n",
    "\n",
    "print('词频在0~99的词数量：',len([w for w in W.values() if w < 100]))\n",
    "\n",
    "print('词频在100~999的词汇量：',len([w for w in W.values() if w >= 100 and w < 1000]))\n",
    "\n",
    "print('词频在1000~4999的词汇量：',len([w for w in W.values() if w > 1000 and w < 5000]))\n",
    "\n",
    "print('词频在5000及其以上的词数量：',len([w for w in W.values() if w >= 5000]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'jieba'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[15], line 2\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mre\u001b[39;00m\n\u001b[1;32m----> 2\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mjieba\u001b[39;00m\n\u001b[0;32m      3\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;28mstr\u001b[39m)\n",
      "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'jieba'"
     ]
    }
   ],
   "source": [
    "import re\n",
    "import jieba\n",
    "print(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
